# -*- codeing = utf-8 -*-
# @Time: 2022/4/23 13:25
# @Author: Foxhuty
# @File: crawling_weather_data.py
# @Software: PyCharm
# @Based on python 3.10
import asyncio
import time
import aiohttp
import aiofiles
import warnings

warnings.filterwarnings('ignore', category=DeprecationWarning)


async def parse_data(url):
    page_json = await get_json(url)
    async with aiofiles.open('new_weather_data.txt', mode='a', encoding='utf-8', newline='') as f:
        for weather in page_json['data']['predict']['detail']:
            city = page_json['data']['predict']['station']['city']
            date = weather['date']
            info = weather['day']['weather']['info']
            temp = weather['day']['weather']['temperature']
            wind = weather['day']['wind']['power']
            wind_direct = weather['day']['wind']['direct']
            print(city, date, info, temp, wind, wind_direct)
            await f.write(f'{city},{date},{info},{temp},{wind},{wind_direct}\n')


async def get_json(url):
    async with aiohttp.ClientSession() as session:
        async with session.get(url, headers=headers, cookies=cookies) as response:
            response.get_encoding()
            page_json = await response.json()
            return page_json


async def main():
    tasks = []
    region_url = "http://www.nmc.cn/rest/province/all?_=1650635275805"
    region_json = await get_json(region_url)
    for region_code in region_json:
        region = region_code['code']
        province = region_code['name']
        if province in ['四川省', '重庆市']:
            url = f"http://www.nmc.cn/rest/province/{region}?_=1650444461514"
            code_json = await get_json(url)
            print(f'==========正在爬取{province}天气数据==========')
            for code in code_json:
                code_number = code['code']
                url_location = f"http://www.nmc.cn/rest/weather?stationid={code_number}&_=1650444461512"
                tasks.append(asyncio.create_task(parse_data(url=url_location, )))
            await asyncio.wait(tasks)
            await asyncio.sleep(5)
        else:
            continue


if __name__ == '__main__':
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36',
        'Referer': 'http://www.nmc.cn/',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Connection': 'keep-alive'
    }
    cookies = {
        "Cookie": "UM_distinctid=18045f807d5196-0798e00eb23f81-4e4c0f20-1fa400-18045f807d6385; __nmcu=1516686969298620416; __utrace=af19b265d6aa7df9474bf2b6e6f20552; ray_leech_token=1650441408; CNZZDATA1254743953=1643250002-1650437482-null%7C1650437504"
    }

    start = time.time()
    asyncio.run(main())
    # loop = asyncio.get_event_loop()
    # loop.run_until_complete(main())
    end = time.time()
    print(f'共计用时{end - start}')
